# -*- coding: utf-8 -*-
"""
Created on Sun May 21 21:39:45 2017

@author: claude
"""
import scrapy
import pymongo
import requests
from xiaohongshu.items import XiaohongshuItem#, XiaohongshuGood, XiaohongshuAlbum

MAX_PAGE_DISCOVERY = 10
MAX_PAGE_SEARCH = 10

class MongoClint():
    collection_item = 'items'
    def __init__(self):
        self.client = pymongo.MongoClient('localhost', 27017)
        self.db = self.client.xiaohongshu
        
    def check_item_in_db(self, item_id):
        if self.db.items.count({"_id":item_id})>0:
            return True
        else:
            return False
            
    def save_res_to_db(self, keywords, id_array):
        dict_item = {}
        dict_item["keyword"] = keywords
        dict_item["ids"] = list(set(id_array))
        try:
            self.db[self.collection_item].insert(dict_item)
        except pymongo.errors.DuplicateKeyError:
            print '_id %s has been existed', dict_item["_id"]
            #logger.debug('_id %s has been existed', dict_item["_id"]) 
    
class SearchSpider(scrapy.Spider):
    """
    Usage: scrapy crawl search -a keywords="金瓶安耐晒"    
    http://m.xiaohongshu.com/web_api/sns/v1/search/note?keyword=%E6%97%A0%E6%AF%94%E6%BB%B4&page=12
    http://m.xiaohongshu.com/web_api/sns/v1/search/note?keyword=cpb&page=5
    """
    name = "search"
    item_url = 'http://www.xiaohongshu.com/discovery/item/'
    base_url = 'http://www.xiaohongshu.com/api/snsweb/v1/search?keyword='
    keyword = None
    mongodb = MongoClint()
        
    def get_keywords(self,keywords):
        arg_string = ""
        if isinstance(keywords, list):
            for word in keywords:
                arg_string += word+"+"
            arg_string = arg_string[:-1]
        else:
            arg_string = keywords
        return arg_string
        
    def __init__(self, keywords=None, *args, **kwargs):
        self.client = pymongo.MongoClient('localhost', 27017)
        self.db = self.client.xiaohongshu        
        super(SearchSpider, self).__init__(*args, **kwargs)
        if keywords==None:
            pass
        else:
            self.keyword = self.get_keywords(keywords)                    
            print self.keyword 

    def start_requests(self):
        id_array = []
        if self.keyword==None:
            id_array.append("545a04c1d6e4a94fa452999d")
        else:
            for page in range(1, MAX_PAGE_SEARCH):
                resp = requests.get(self.base_url+self.keyword+'&mode=word_search&page='+repr(page))
                js = resp.json()
                if len(js["data"]["notes"])<2:
                    self.logger.debug('Hi, total page %s', page)
                    break
                else:
                    for item in js["data"]["notes"]:
                        id_array.append(item["id"])    
        for item_id in id_array:    
            if self.mongodb.check_item_in_db(item_id)==False:
                yield scrapy.Request(url=self.item_url+item_id, callback=self.parse_item)

#################### parse item ####################
    def parse_item(self, response):
        """
        item:       note, URL=/discovery/item/*
        good:      goods, URL=/goods/*
        album:    album, URL=/selected_board/*
        """
        self.logger.info('Hi, this is an ITEM page! %s', response.url)
        item = XiaohongshuItem()
        self.get_item_id(response,item)
        self.get_info(response,item)
        self.get_item_product(response,item)
        self.get_item_description(response,item)
        self.get_item_recommendation(response,item)
        self.get_item_related_note(response,item)
        self.get_item_album(response,item)
        yield item
        
    def get_related_discoveries(self, response):
        """
        sub function in parse_item 
        http://www.xiaohongshu.com/api/snsweb/v1/get_discovery_related_discoveries?page=1&tag_id=&discovery_id=5703a14c78362338b6d66328
        """
        related_items_id = (response.url).split("/")[-1]
        id_array = []
        for page in range(1, MAX_PAGE_DISCOVERY):
            related_items_url = "http://www.xiaohongshu.com/api/snsweb/v1/get_discovery_related_discoveries?page=%d&tag_id=&discovery_id=%s" % (page, related_items_id)            
            self.logger.info('get_related_discoveries, %s', related_items_url)            
            resp = requests.get(related_items_url)
            js = resp.json()
            if len(js["data"])<2:
                break
            else:
                for item in js["data"]:
                    id_array.append(item["id"]) 
        return id_array
        
    def get_item_id(self, response, item):
        item['_id'] = response.url.split("/")[-1]
        
    def get_info(self, response, item):
        """
        author, author_id, title, collect_score, like_score, comments_score, picture_url
        """
        section_1 = response.xpath('/html/body/section[1]')
        item['author'] = section_1.xpath('//div[2]/div[2]/a/text()').extract_first()
        item['author_id'] = section_1.xpath('//div[2]/div[2]').css('a::attr(href)').extract_first()
        item['title'] = section_1.xpath('//div[3]/h2/text()').extract_first()
        s = section_1.xpath('//div[4]').css('span::text').extract()    
        if len(s)>=2:
            if len(s[0])>2:
                item['collect_score'] = int(s[0].split(u'\xb7')[-1])
            else:
                item['collect_score'] = 0
            if len(s[1])>1:
                item['like_score'] = int(s[1].split(u'\xb7')[-1])
            else:
                item['like_score'] = 0
        else:
            item['like_score'] = -1
            item['collect_score'] = -1
        s = section_1.xpath('//div[5]/a/em/text()').extract_first()
        if s == None:
            item['comments_score'] = 0
        else:
            item['comments_score'] = int()
        pic_array = []
        for pic in section_1.css("img.image::attr(data-src)").extract():
            pic_array.append(pic)
        item['picture_url'] = pic_array

    def get_item_product(self, response, item):
        item['product'] = response.xpath('/html/body/section[2]/*[@id="note_item"]/div/div[2]/h1/text()').extract_first()        
    def get_item_description(self, response, item):
        texts = ""
        maintext = response.css("div.note-desc")
        for sentence in maintext.css("p.content::text").extract():
            texts += sentence
        item['description'] = texts
    def get_item_recommendation(self, response, item):
        item['recommendation'] = response.xpath('/html/body/section[3]/ul').css('a::attr(href)').extract()
    def get_item_related_note(self, response, item):
        item['related_note']  = response.xpath('/html/body/section[4]/*[@id="note_list"]').css('a::attr(href)').extract()
        id_array = self.get_related_discoveries(response)
        item['related_note'] += id_array
    def get_item_album(self, response, item):
        item['album']  = response.xpath('/html/body/section[5]').css('a.board::attr(href)').extract()
####################### end ######################
        